'''
Created on Jan 8, 2012

@author: haoyu
'''

import os
import string
import time
import codecs

def loaddata():
  root_path = "/Users/haoyu/Documents/password_db/csdn/"
  filename = os.path.join(root_path,"csdnpassport_utf8.txt")
  c = 0
  ce = 0
  dataset = {}
  starttime = time.time()
  for row in open(filename):
    c += 1
    row = row.strip()
    if not row:
      continue
    id,username,password,email,sdate,stime,encypt_pwd = row.split()[:7]
    if username[0].lower() in string.lowercase[:16]:
      dataset[username] = password
  
  endtime = time.time()
  print "elapse time:%s,rows=%s" % ((endtime-starttime),len(dataset))
  
  
  hackfile = os.path.join(root_path,"user_from_hack.txt")
  c = 0
  sa = 0
  for row in codecs.open(hackfile,"r","utf-16"):
    
    row = row.strip()
    if not row:
      contine
    row = row.encode("utf-8")
    try:
      id,username,password,email = row.split()[:4]
      if username[0].lower() in string.lowercase[:16]: 
        c += 1
        if dataset.has_key(username) and password==dataset[username]:
          sa += 1
    except:
      pass
  print "c:%s,sa:%s,datacount:%s" % (c,sa,len(dataset))
    
def test():
  root_path = "/Users/haoyu/Documents/password_db/csdn/"
  filename = os.path.join(root_path,"CsdnPassport.txt")
  newfile = os.path.join(root_path,"csdnpassport_utf8.txt")
  c = 0
  ce = 0
  dataset = []
  nf = open(newfile,"w")
  for row in codecs.open(filename,"r","utf-16"):
    c += 1
    row = row.strip()
    row = row.encode("utf8")
    #row = row.decode
    if not row:
      continue
    #print row.split()
    #print row
    
    #for f in row.split():
    #  print f
      
    
    try:
      
      id,username,password,email,sdate,stime,encypt_pwd = row.split()[:7]
      nrow = "\t".join(row.split()[:7])
      dataset.append("%s\n" % nrow)
      if c % 1000 ==0:
        nf.writelines(dataset)
        dataset = []
    except Exception,ex:
      print ex      
      ce += 1
    
  nf.close()


def splitfile():
  root_path = "/Users/haoyu/Documents/password_db/csdn"
  filename = os.path.join(root_path,"CsdnPassport.txt")
  allletter = ""
  starttime = time.time()
  
  fs ={}
  for char in string.ascii_lowercase:
    allletter += char
    split_filename = os.path.join(root_path,"split_csdnpassport","csdnpassport_%s.txt" % char)
    if not os.path.exists(split_filename):
      open(split_filename,"w").write("")
  
    fs[char] =  open(split_filename,"a")
    
  print allletter
  #return 
  for row in codecs.open(filename,"r","utf-16"):
    row = row.strip()

    if not row:
      continue
    row = row.encode("utf8")
    
    print row
    id,username,password,email,sdate,stime,encypt_pwd = map(lambda x:x.strip(),row.split())
    
    char = username.lower()[0:2]
    #print username.decode("uu_codec")
    print "|%s|" % username,len(username),"|%s|" % char
    if char in allletter:       
      fs[char].write("%s\n" % row)
  
    if username == u"a20070918":
      print username
      return 
  endtime = time.time()
  print "elapse time:%s" % (endtime - starttime)
          
if __name__ == "__main__":
  loaddata()